//////////////////////////////////////////////////////////////////////////
//
// This file is a part of XORT project.
//
// PROJECT:         Xort compiler
// FILE:            xort.h
// AUTHOR:          Vladimir Gumenuk
// DATE:            19-Mar-2008
// MODIFICATIONS:   28-11-2012: reviewed and added to svn
// 
// NOTES: The coding style of this program is specially chosen for 1 purpose:
// to be close as much as possible to the target language. This should allow
// the compiler to compile itself. 
//
//
//     Copyright(c) 2014, Vladimir Gumenuk
//     All rights reserved.
//
//     Redistribution and use in source and binary forms, with or without modification, 
//     are permitted provided that the following conditions are met :
//
//     1. Redistributions of source code must retain the above copyright notice, 
//        this list of conditions and the following disclaimer.
//     2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and 
//        the following disclaimer in the documentation and / or other materials provided with the distribution.
//     3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse 
//        or promote products derived from this software without specific prior written permission.
//
//     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
//     ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 
//     OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
//     THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
//     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 
//     OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
//     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
//     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 
//     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#pragma once

#ifdef CPP_COMPILER
    #ifdef BUILD_FOR_PC
        #pragma warning(disable:4244) // conversion from 'type1' to 'type2', possible loss of data
        #pragma warning(disable:4146) // unary minus operator applied to unsigned type, result still unsigned
        #pragma warning(disable:4996) // 'strdup': The POSIX name for this item is deprecated.
        #include <assert.h>
        #include <stdio.h>
        #include <memory.h>
        #include <string.h>
        #include <stdarg.h>
        #include <stdlib.h>
        #include <direct.h>
        #ifdef __XVM__
            #include <unknwn.h>
        #endif

    #elif defined(BUILD_FOR_MAC)
        #include <assert.h>
        #include <stdio.h>
        #include <stdarg.h>
        #include <stdlib.h>

        struct IUnknown {
            virtual long QueryInterface() = 0;
            virtual long AddRef() = 0;
            virtual long Release() = 0;
        };
        typedef IUnknown* LPUNKNOWN;

        char* _itoa(int value, char* result, int base);
        int _vscprintf(const char * format, va_list pargs);
    #endif
        
    #include "xort_sdk.h"

    struct XortElement;
    class XRuntime;
    struct Symbol;
    typedef Symbol* symbolref;

    #define IMPLEMENT_XUNKNOWN() \
        virtual dword AddRef() { return ++m_dwRefs; } \
        virtual dword Release() { if( --m_dwRefs == 0 ) { Uninit(); delete this; return 0; } return m_dwRefs; } \
        virtual dword GetRefCount() { return m_dwRefs; }

    struct xobject : xunknown
    {
        dword    m_dwRefs;
        dword    m_dwType; // custom type assigned by compiler

        xobject() { m_dwRefs = 0; m_dwType = 0; }
        virtual ~xobject() {
            if( m_dwRefs != 0 ) {
               // TODO: error
            }
        }

        IMPLEMENT_XUNKNOWN();

        void SetType( dword dwType ) { m_dwType = dwType; }
                
        // XVM runtime support:
        // builds the object fields
        virtual int Init( symbolref pSymbol, XortNativeInterface *pExternalObj ) = 0;

        // cleans the object fields
        virtual void Uninit() { ; }

        virtual dword GetType() { return m_dwType; }

        // Invokes the object method
        virtual int Invoke( dword dwFuncIndex, XRuntime *pRuntime ) = 0;

        // assigns new value to the internal variable
        virtual int Put( dword dwVarIndex, const XortElement& Value ) = 0;

        // returns the value of the internal variable
        virtual int Get( dword dwVarIndex, XortElement& Value ) = 0;

        // returns the count of internal variables;
        virtual dword GetVariablesCount() = 0;

        // used internally by XVM
        virtual symbolref get_Symbol() { return 0; }                 // returns pointer to symbol
        virtual symbolref get_Method( dword dwIdx ) { return 0; }    // pointer to method
        virtual int Pack( void **ppOut ) { return 0; }               // returns the ByteArray object with packed data
        virtual int Unpack( void *pByteArray ) { return 0; }         // accepts the ByteArray object with packed data 
        virtual int CreateBinaryObject( void **ppOut ) { return 0; } // returns the ByteArray object with packed data ready for jit
    };

    void DbgPrint( const char *lpszFmt, ... );

    typedef xobject* xobjectref;

    #define NULL 0
    #define TRUE 1
    #define FALSE 0
    #define DEREF(x) (*x)
    #define MEMBER_PTR(x) x->
    #define ADDROF(x)  &x
    #define STATIC  static
    #define EOFSTRUCT ;
    #define CONST_CHAR_PTR (const char*)
    #define NEW_OBJECT(x) = new x();
    #define PTR *
    #define CONST const
#else
    typedef long                qlong;
    const int TRUE = 1;
    const int FALSE = 0;

    #define DEREF(x) x
    #define MEMBER_PTR(x) x.
    #define ADDROF(x)  x
    #define STATIC
    #define EOFSTRUCT 
    #define CONST_CHAR_PTR
    #define NEW_OBJECT(x)
    #define PTR 
    #define CONST

[import("proc") name("printf") module("msvcrt.dll") callconv("cdecl") platform("windows")]
int printf( string csFormat, ... );
/*
[import("proc") name("printf") module("libsystem_c.dylib") callconv("cdecl") platform("osx")]
int printf( string csFormat, ... );
*/

#endif

typedef double              real;

//////////////////////////////////////////////////////////////////////////
enum PrimCodes
{
    p_addrof = 0x01,
    p_load = 0x02,
    p_mov = 0x03,
    p_store = 0x04,
    p_drop = 0x05,
    p_dup = 0x06,
    p_swap = 0x07,
    p_or = 0x08,
    p_xor = 0x09,
    p_and = 0x0A,
    p_shl = 0x0B,
    p_shr = 0x0C,
    p_add = 0x0D,
    p_sub = 0x0E,
    p_mul = 0x0F,
    p_div = 0x10,
    p_mod = 0x11,
    p_inc = 0x12,
    p_dec = 0x13,
    p_neg = 0x14,
    p_not = 0x15,
    p_nz = 0x16,
    p_lor = 0x17,
    p_land = 0x18,
    p_lnot = 0x19,
    p_cmpeq = 0x1A,
    p_cmpneq = 0x1B,
    p_cmpgt = 0x1C,
    p_cmplt = 0x1D,
    p_cmpge = 0x1E,
    p_cmple = 0x1F,
    p_convert = 0x20,
    p_goto = 0x21,
    p_ifnz = 0x23,
    p_ifz = 0x22,
    p_ifeq = 0x24,
    p_ifneq = 0x25,
    p_ifleq = 0x26,
    p_ifgeq = 0x27,
    p_ifg = 0x28,
    p_ifl = 0x29,
    p_calli = 0x2A,
    p_call = 0x2B,
    p_ret = 0x2C,
    p_new = 0x2D,
    p_get = 0x2E,
    p_put = 0x2F,
    p_invoke = 0x30,
    p_gettype = 0x31,
    p_enter = 0x32,
    p_leave = 0x33,
    p_nop = 0x34,
    p_dbgbreak = 0x35,
    p_pack = 0x36,
    p_upack = 0x37,
    p_geti = 0x38, // next immediate value is always word
    p_puti = 0x39,
    p_invokei = 0x3A,
}EOFSTRUCT

//////////////////////////////////////////////////////////////////////////
enum PrimTypeCodes
{
    t_v  = 0x00,
    t_s1 = 0x10, // 0C
    t_u1 = 0x20, // 0B
    t_s2 = 0x30, // 0S
    t_u2 = 0x40, // 0W
    t_s4 = 0x50, // 0I
    t_u4 = 0x60, // 0U
    t_s8 = 0x70, // 0L
    t_u8 = 0x80, // 0Q
    t_r4 = 0x90, // 0F
    t_r8 = 0xA0, // 0R
    t_p  = 0xB0,
}EOFSTRUCT

//////////////////////////////////////////////////////////////////////////
enum PrimAddrOperandCodes
{
    a_0 = 0x00,
    a_c = 0x01,
    a_m = 0x02,
    a_s = 0x03,
    a_indm = 0x04,
    a_inds = 0x05,
    a_cs = 0x06,
    a_cm = 0x07,
    a_ms = 0x08,
    a_sm = 0x09,
    a_ss = 0x0A,
    a_mm = 0x0B,
    a_ma = 0x0C,
    a_sa = 0x0D,
    a_ta = 0x0E,
    a_ti = 0x0F,
}EOFSTRUCT

//////////////////////////////////////////////////////////////////////////
// errors runtime
const int XORT_OK                       = 1;
const int ERR_UNKNOWN_PRIMCODE          = -1;
const int ERR_INVALID_PRIMCODE          = -2;
const int ERR_INVALID_STACK_SYSFRAME    = -3;
const int ERR_THIS_PTR_NULL             = -4;
const int ERR_MEMORY_ALLOCATION         = -5;
const int ERR_INVALID_SYMBOL_KIND       = -6;
const int ERR_ASSIGN_TO_CONST_PROP      = -7;
const int ERR_SYNTAX                    = -8;
const int ERR_INTERNAL                  = -9;
const int ERR_INVALID_TYPE              = -10;
const int ERR_TYPES_MISMATCH            = -11;
const int ERR_CONVERSION_NOTPOSSIBLE    = -12;
const int ERR_PRIMITIVE_NOTIMPL         = -13;
const int ERR_TYPE_NOT_FOUND            = -14;
const int ERR_RAWPTR_TYPE_FOUND         = -15;
const int ERR_INVALID_STORAGE           = -16;
const int ERR_INVALID_PROPERTY_INDEX    = -17;
const int ERR_INVALID_METHOD_INDEX      = -18;
const int ERR_STACK_UNDERFLOW           = -19;
const int ERR_INVALID_ARG               = -20;
const int ERR_INVALID_ARG_COUNT         = -21;
const int ERR_INVALID_ACCESS            = -22;
const int ERR_INVALID_ADDR              = -23;
const int ERR_LABEL_NOT_FOUND           = -24;
const int ERR_DEAD_LOOP_DETECTED        = -25;
const int ERR_JUMP_OFFSET               = -26;
const int ERR_UNUSED_LABEL_FOUND        = -27;
const int ERR_INVALID_STACK             = -28;
const int ERR_SYMBOL_NOT_PROC           = -29;
const int ERR_METHOD_NOT_IMPL           = -30;
const int ERR_LOAD_LIBRARY_FAIL         = -31;
const int ERR_GET_PROC_ADDR_FAIL        = -32;
const int ERR_CALLCONV_NOT_SUPPORTED    = -33;
const int ERR_ENTRYPOINT_NOT_FOUND      = -34;
const int ERR_XVM_TERMINATE             = -35;
const int ERR_DBG_INVALID_FRAMESCOPE    = -36;
const int ERR_METHOD_NOT_FOUND          = -37;
const int ERR_UNSUPPORTED_TYPE_FOR_NATIVE_CALL = -38;
const int ERR_ZERO_SIZE_ARRAY           = -39;
const int ERR_PROC_NOT_IMPLEMENTED      = -40;
const int ERR_INVALID_OBJECT_NAME       = -41;
const int ERR_INVALID_ATTRIBUTE         = -42;
const int ERR_NULL_PTR                  = -43;
const int ERR_INVALID_DATA_IDX          = -44;
const int ERR_INVALID_STACK_IDX         = -45;

// compilation errors
const dword  ERR_INTERNAL_COMPILER              = 0;
const dword  ERR_UNDECLARED_IDENTIFIER          = 1;
const dword  ERR_UNEXPECTED_TOKEN               = 2;
const dword  ERR_UNEXPECTED_EOF                 = 3;
const dword  ERR_UNKNOWN_TOKEN                  = 4;
const dword  ERR_UNKNOWN_TYPE                   = 5;
const dword  ERR_IMPOSSIBLE_CONVERSION          = 6;
const dword  ERR_IDENT_NOT_FOUND                = 7;
const dword  ERR_IDENT_ISVARORPROC              = 8;
const dword  ERR_DECLARAYION_SYNTAX             = 9;
const dword  ERR_CONST_DECLARAYION              = 10;
const dword  ERR_CONST_EXPR_SYNTAX              = 11;
const dword  ERR_EXPR_SYNTAX                    = 12;
const dword  ERR_STRING_SYNTAX                  = 13;
const dword  ERR_FLOATINGP_SYNTAX               = 14;
const dword  ERR_MODOFIER_APPLIED_SYNTAX        = 15;
const dword  ERR_NESTED_PROC                    = 16;
const dword  ERR_OPERATION_NOT_ALLOWED          = 17;
const dword  ERR_INCDEC_NOT_SUPP_FORCOMPLEX     = 18;
const dword  ERR_INCDEC_NOT_SUPP_FORTHISTYPE    = 19;
const dword  ERR_UNARY_MINUS_NOT_ALLOWED        = 20;
const dword  ERR_UNARY_NOT_NOT_ALLOWED          = 21;
const dword  ERR_UNARY_LOGNOT_NOT_ALLOWED       = 22;
const dword  ERR_ELSE_WITHOUT_IF                = 23;
const dword  ERR_EXPECTED_BLKCLOSE              = 24;
const dword  ERR_EXPECTED_IDENT                 = 25;
const dword  ERR_EXPECTED_CONST_EXPR            = 26;
const dword  ERR_EXPECTED_TYPENAME              = 27;
const dword  ERR_EXPECTED_ARG_NAME              = 28;
const dword  ERR_EXPECTED_SINGLE_LINE_STMT      = 29;
const dword  ERR_EXPECTED_OPERATOR_ASSIGN       = 30;
const dword  ERR_MISSING_PARENTHOPEN            = 31;
const dword  ERR_MISSING_PARENTHCLOSE           = 32;
const dword  ERR_MISSING_BLKOPEN                = 33;
const dword  ERR_MISSING_BLKCLOSE               = 34;
const dword  ERR_MISSING_SEMICOLON              = 35;
const dword  ERR_MISSING_COMA                   = 36;
const dword  ERR_MISSING_MISSING_GREATER        = 37;
const dword  ERR_MISSING_QUOTE                  = 38;
const dword  ERR_MISSING_WHILE                  = 39;
const dword  ERR_IDENT_ISNOT_VAR                = 40;
const dword  ERR_VOID_FUNC_RETURN_VAL           = 41;
const dword  ERR_INSUFF_NUM_ARGS                = 42;
const dword  ERR_INVALID_NUM_PARAMS             = 43;
const dword  ERR_TOOMANY_FMT_STRS               = 44;
const dword  ERR_OUTSIDE_STRING_UNSPECIF_NUM_PARAMS = 45;
const dword  ERR_CANNOT_ASGN_TO_PROC            = 46;
const dword  ERR_ARG_ALREADY_DECL_WITH_DIFTYPE  = 47;
const dword  ERR_ARG_ALREADY_DECL               = 48;
const dword  ERR_ARG_FORWARD_DECL_DIFFERS       = 49;
const dword  ERR_FUNC_DECL_ERROR                = 50;
const dword  ERR_CONST_EXPR_TYPES_MISMATCH      = 51;
const dword  ERR_IDENT_ALREADY_DECLARED         = 52;
const dword  ERR_FUNC_DECLARED_WITH_DIFF_RETTYPE= 53;
const dword  ERR_VOID_FUNC_CANNOTBE_PART_EXPR   = 54;
const dword  ERR_UNARY_MINUS_APPLIED_TOUNSIGNED = 55;
const dword  ERR_INTERNAL_DECODE_INSTR          = 56;
const dword  ERR_DEFAULT_ALREADY_GOT            = 57;
const dword  ERR_MISSING_COLON                  = 58;
const dword  ERR_CASE_WITHOUT_SWITCH            = 59;
const dword  ERR_STATIC_INITIALIZERS            = 60;
const dword  ERR_HEXDIGIT_SYNTAX                = 61;
const dword  ERR_ENUM_SYNTAX                    = 62;
const dword  ERR_LVALUE_IS_CONSTANT             = 63;
const dword  WRN_SECOND_OBJ_IGNORED             = 64;
const dword  ERR_MISSING_BREAK                  = 65;
const dword  ERR_EXPECTED_ATTRIBUTE             = 66;
const dword  ERR_UNKNOWN_ATTRIBUTE              = 67;
const dword  ERR_ATTRIBUTE_DECL                 = 68;
const dword  ERR_PROC_DECL_EXPECTED             = 69;
const dword  ERR_FORWARD_PROC_WITHATTR          = 70;
const dword  ERR_CANT_READ_FILE                 = 71;
const dword  ERR_INVALID_PLATFORM               = 72;
const dword  ERR_TYPE_NATIVELY_NOT_ALLOWED      = 73;
const dword  ERR_TYPE_NOT_ALLOWED_IN_EXPORT_OBJ = 74;

const dword MARKER_PROC            = 0x434F5250; // 'PROC'
const dword MARKER_STRU            = 0x55525453; // 'STRU'
const dword MARKER_BOBJ            = 0x4A424F42; // 'BOBJ'

const dword t_void    = 0;              // defined for compiler only!!!
// the rest is defined in xort prims

const byte  p_beginarg  = 0xEE;         // metainstruction to mark function argument calculation
const byte  p_endarg    = 0xEF;
const byte  p_begswitch = 0xF1;          // pseudo instruction to mark switch value calculation
const byte  p_endswitch = 0xF2;
const byte  p_begcase   = 0xF3;          // pseudo instruction to mark case test
const byte  p_endcase   = 0xF4;
const byte  p_beginret  = 0xF5;          // pseudo instruction to mark return expression
const byte  p_endret    = 0xF6;
const byte  p_beginasgn = 0xF7;          // pseudo instruction, used to mark assignment expression
const byte  p_endasgn   = 0xF8;          //                                  
const byte  p_begin_if  = 0xF9;          // pseudo instruction to mark conditional statements (if, while, for)
const byte  p_end_if    = 0xFA;
const byte  p_begincall = 0xFB;           // pseudo instruction, used to mark begin of procedure call
const byte  p_endcall   = 0xFC;           //                                  end of procedure call
const byte  p_stkvar    = 0xFD;           // pseudo instruction used to mark stack variable. Length - 3 bytes: 0xFE 00 00 - contains index of the variable    
const byte  p_endstkvar = 0xFE;           // pseudo instruction, used to mark the end of sequence related to generating stack local variable
const byte  p_label     = 0xFF;           // pseudo command used to mark labels. Removed on final pass

// build-in system types
const dword t_system   = 0xF0000000;     // system object starts from this!
const dword t_string   = 0xF0000001;
const dword t_array    = 0xF0000002;
const dword t_list     = 0xF0000003;
const dword t_listiter = 0xF0000004;
const dword t_file     = 0xF0000005;
const dword t_xelement = 0xF0000006; // XortElement type
const dword t_userobj  = 0x100;          // user defined types start from this

//////////////////////////////////////////////////////////////////////////
// expression modifiers, and other flags
const dword f_const         = 0x00000001;
const dword f_enumconst     = 0x00000002;
const dword f_unspecnumargs = 0x00000004; // for function with unspecified number of arguments
const dword f_proc_expected = 0x00000008; // means, that compiler expecting procedure declaration
const dword f_import_proc   = 0x00000010; // indicate that procedure must be imported from some native dll
const dword f_buildinfunc   = 0x00000020; // for build-in function
const dword f_import_obj    = 0x00000040; // indicate, that object should be loaded from some native dll
const dword f_allowed_native= 0x00000080; // indicate, that this type is allowed in native imported object declaration
const dword f_export_obj    = 0x00000100; // indeicate that this object is exported from XVM to native dll
const dword f_procexecuted  = 0x10000000; // set by runtime, to indicate that procedure has already been executed
const dword f_labels_removed= 0x20000000; // set by runtime, to indicate, that labels has already been removed 
const dword f_iplinemap_gen = 0x40000000; // set by runtime, to indicate, that ip-to-line has already been mapped
const dword f_instr_decoded = 0x80000000; // set by runtime, to indicate, that all instructions have been decoded for this symbol

//////////////////////////////////////////////////////////////////////////
// symtab.cpp
// symbol kind codes
const dword sk_nullsymbol  = 0; // special case: NULL symbol
const dword sk_scopeglobal = 1; // global namespace marker
const dword sk_scopelocal  = 2; // local namespace marker
const dword sk_var         = 3; // variable
const dword sk_arg         = 4; // argument
const dword sk_obj         = 5; // object
const dword sk_proc        = 6; // procedure
const dword sk_typename    = 7; // build-in types
const dword sk_template    = 8; // build-in template class
const dword sk_integer     = 9; // constants
const dword sk_real        = 10;// floating point number
const dword sk_nullptr     = 11;// special type for null pointers
const dword sk_stackvar    = 12;// special case, returned by code generator

// token types
const dword tt_unknown   = 0; // unrecognized token
const dword tt_delim     = 1;
const dword tt_typename  = 2;
const dword tt_ident     = 3;
const dword tt_digit     = 4;
const dword tt_literal   = 5;
const dword tt_stmt      = 6;
const dword tt_modifier  = 7;
const dword tt_template  = 8;
const dword tt_eof       = 9;
const dword tt_null      = 10; // special case for NULL pointers
const dword tt_attr      = 11; // for attributes support

// token codes (token kind)
const dword tk_unknown      = 0;
const dword tk_if           = 1;   // commands
const dword tk_else         = 2;
const dword tk_while        = 3;
const dword tk_do           = 4;
const dword tk_break        = 5;
const dword tk_object       = 6;
const dword tk_continue     = 7;
const dword tk_typename     = 8;
const dword tk_return       = 9;
const dword tk_ident        = 10;  // identifiers
const dword tk_number       = 11;
const dword tk_realnumber   = 12;
const dword tk_blkopen      = 13;  // delimiters
const dword tk_blkclose     = 14;
const dword tk_parenthopen  = 15;
const dword tk_parenthclose = 16;
const dword tk_sqrbropen    = 17;
const dword tk_sqrbrclose   = 18;
const dword tk_coma         = 19;
const dword tk_semicolon    = 20;
const dword tk_assign       = 21;  // operators
const dword tk_assignminus  = 22;
const dword tk_assignplus   = 23;
const dword tk_assignmul    = 24;
const dword tk_assigndiv    = 25;
const dword tk_assignmod    = 26;
const dword tk_assignshr    = 27;
const dword tk_assignshl    = 28;
const dword tk_assignbinand = 29;
const dword tk_assignxor    = 30;
const dword tk_assignbinor  = 31;
const dword tk_plus         = 32;
const dword tk_minus        = 33;
const dword tk_mul          = 34;
const dword tk_div          = 35;
const dword tk_mod          = 36;
const dword tk_inc          = 37;
const dword tk_dec          = 38;
const dword tk_greater      = 39;
const dword tk_less         = 40;
const dword tk_equal        = 41;
const dword tk_notequal     = 42;
const dword tk_greaterequal = 43;
const dword tk_lessequal    = 44;
const dword tk_not          = 45;
const dword tk_quest        = 46;
const dword tk_colon        = 47;
const dword tk_logor        = 48;
const dword tk_logand       = 49;
const dword tk_binor        = 50;
const dword tk_binand       = 51;
const dword tk_xor          = 52;
const dword tk_shiftleft    = 53;
const dword tk_shiftright   = 54;
const dword tk_tilda        = 55;
const dword tk_ptr          = 56;
const dword tk_const        = 57;
const dword tk_for          = 58; 
const dword tk_typedef      = 59;
const dword tk_array        = 60;
const dword tk_list         = 61;
const dword tk_switch       = 62;
const dword tk_case         = 63;
const dword tk_default      = 64;
const dword tk_enum         = 65;
const dword tk_nullptr      = 66; // :)
const dword tk_elipsis      = 67; // ...
const dword tk_a_import     = 68;
const dword tk_a_name       = 69;
const dword tk_a_module     = 70;
const dword tk_a_callconv   = 71;
const dword tk_a_platform   = 72; // temporary solution to support Mac OS, Linux and iOS... :)
const dword tk_a_export     = 73; // attribute export

//////////////////////////////////////////////////////////////////////////
// utils.cpp
byte   x_lobyte( word w );
byte   x_hibyte( word w );
word   x_loword( dword dw );
word   x_hiword( dword dw );
short  x_makeShort( char b1, char b2 );
word   x_makeWord( byte b1, byte b2 );
int    x_makeInt( short s1, short s2 );
dword  x_makeDword( word w1, word w2 );
qlong  x_makeLong( int s1, int s2 );
qword  x_makeQword( dword w1, dword w2 );
dword  x_tok_from_code( byte bCode );

struct SymbolListInfo
{
    dword dwTotalSymbolsCount;
    dword dwVariblesCount;
    dword dwFunctionsCount;
}EOFSTRUCT

//////////////////////////////////////////////////////////////////////////
//
#ifdef CPP_COMPILER
    #include "XortString.h"
    #include <map>
    //#define DEBUG_CLIENTBLOCK   new( _CLIENT_BLOCK, __FILE__, __LINE__)
    //#define _CRTDBG_MAP_ALLOC
    //#include <crtdbg.h>
    //#define new DEBUG_CLIENTBLOCK
    
#pragma pack(1)
    struct XortElement {
        byte bType;
        byte bAccess;
        union {
            struct {
                dword _asp; // address of the parent stack frame
                dword _ti;  // this pointer storage for VM
            };
            char    _s1;
            byte    _u1;
            short   _s2;
            word    _u2;
            int     _s4;
            dword   _u4;
            qlong   _s8;
            qword   _u8;
            float   _r4;
            real    _r8;
            xobject* _p;
        };
        XortElement() {
            bType   = 0;
            bAccess = 0;
            _u8     = 0;
        }
        XortElement( const XortElement& x ) {
            bType = x.bType;
            bAccess = x.bAccess;
            _u8 = x._u8;
            if( bType == t_p && _p )
                _p->AddRef();
        }
        ~XortElement() {
            Clear();
        }
        void operator= ( const XortElement& x ) {
            if( &x == this ) return;
            Clear();
            bType = x.bType;
            bAccess = x.bAccess;
            _u8 = x._u8;
            if( bType == t_p && _p )
                _p->AddRef();
        }
        void operator= ( int x ) { // :)
            Clear();
            bType = (byte)x;
            bAccess = (byte)x;
            _u8 = (qword)x;
        }
        void Clear() {
            if( bType == t_p && _p )
                _p->Release();
            bType = 0;
            bAccess = 0;
            _u8 = 0;
        }
        void SetValue( const XortElement& x ) {
            if( &x == this ) return;
            Clear();
            bType = x.bType;
            bAccess = x.bAccess;
            _u8 = x._u8;
            if( bType == t_p && _p )
                _p->AddRef();
        }
#ifdef __XVM__
        int InitFromXNIVariant( const XNIVariant& vVal, symbolref pSymType, XRuntime *pRuntime );
        int CopyToXNIVariant( XNIVariant& vVal, symbolref pSymType ) const;
#endif

        inline operator char() const { return _s1; }
        inline operator byte() const { return _u1; }
        inline operator short() const { return _s2; }
        inline operator word() const { return _u2; }
        inline operator int() const { return _s4; }
        inline operator dword() const { return _u4; }
        inline operator qlong() const { return _s8; }
        inline operator qword() const { return _u8; }
        inline operator float() const { return _r4; }
        inline operator real() const { return _r8; }
        inline operator xobjectref() const { return _p; }

        inline void SetValue( char  val ) { _s1 = val; }
        inline void SetValue( byte  val ) { _u1 = val; }
        inline void SetValue( short val ) { _s2 = val; }
        inline void SetValue( word  val ) { _u2 = val; }
        inline void SetValue( int   val ) { _s4 = val; }
        inline void SetValue( dword val ) { _u4 = val; }
        inline void SetValue( qlong val ) { _s8 = val; }
        inline void SetValue( qword val ) { _u8 = val; }
        inline void SetValue( float val ) { _r4 = val; }
        inline void SetValue( real  val ) { _r8 = val; }
        inline void SetValue( xobjectref val ) { if( bType >= t_p && _p ){ _p->Release(); _p = 0; } _p = val; if( _p ) _p->AddRef(); }
    };
#pragma pack()

#define sys_Frame   0x80        // 1000 0000
#define sys_Scope   0x40        // 0100 0000
#define usr_VarR    0x02        // 0000 0010
#define usr_VarRW   0x06        // 0000 0110

#define IMPLEMENT_SYSTEM_XUNKNOWN() \
    virtual int Init( symbolref pSymbol, XortNativeInterface *pExternalObj ) { return XORT_OK; } \
    virtual dword GetType() { return t_system; }; \
    virtual int Invoke( dword dwFuncIndex, XRuntime *pRuntime ) { return ERR_METHOD_NOT_IMPL; } \
    virtual int Put( dword dwVarIndex, const XortElement& Value ) { return ERR_METHOD_NOT_IMPL; } \
    virtual int Get( dword dwVarIndex, XortElement& Value ) { return ERR_METHOD_NOT_IMPL; } \
    virtual dword GetVariablesCount() { return 0; }

    class InstructionData : public xunknown
    {
        dword m_dwRefs;
    public:
        InstructionData() { m_dwRefs = 0; m_bInstrCode = 0; m_bTypeAddr = 0; m_dwShiftToNext = 0; m_dwLineNumber = 0; }
        virtual ~InstructionData() { }
        IMPLEMENT_XUNKNOWN();
        void Uninit() { m_bInstrCode = 0; m_bTypeAddr = 0; m_dwShiftToNext = 0; m_dwLineNumber = 0; m_Value.Clear(); m_Value2.Clear(); m_strInstr.Empty(); }
        byte        m_bInstrCode;       // instruction code
        byte        m_bTypeAddr;        // address-operand byte
        XortElement m_Value;            // data for the instruction
        XortElement m_Value2;           // data for the instruction
        string      m_strInstr;         // human readable form of the instruction
        dword       m_dwShiftToNext;    // how many bytes is needed to shift to the next instruction
        dword       m_dwLineNumber;     // the number of line in the text file fromwhere this instruction was generated
    };

    // primitive
    struct XPrim : public xobject
    {
        virtual ~XPrim() { ; }
        IMPLEMENT_SYSTEM_XUNKNOWN()
        byte   m_bCode;
        byte   m_bAddrType;
        string m_csAsm;
    };

    #include "list.h"

    // attributes support. example: [import("proc") name("fopen") module("msvcrt.dll") callconv("cdecl")]
    struct Attributes : public xobject
    {
        virtual ~Attributes() { ; }
        IMPLEMENT_SYSTEM_XUNKNOWN()
        dword  dwAttrCode;
        string csAttrValue;
    };
        
    typedef xlist<symbolref, CCopyObject<symbolref>, CDeleteObject<symbolref> >      SYMBOL_LIST;
    typedef ByteArray                                                                BYTE_ARRAY;
    typedef xarray<int, CCopySimpleType<int>, CDeleteSimpleType<int> >               INT_ARRAY;
    typedef xarray<dword, CCopySimpleType<dword>, CDeleteSimpleType<dword> >         DWORD_ARRAY;
    typedef xarray<stringref, CCopyObject<stringref>, CDeleteObject<stringref> >     STRING_ARRAY;

    // for runtime subsystem
    typedef xstack                                                                   XCELLSARRAY; 
    typedef xarray< symbolref, CCopyObject<symbolref>, CDeleteObject<symbolref> >    SYMBOLSARRAY;
    typedef xarray< XNIVariant, CCopySimpleType<XNIVariant>, CDeleteSimpleType<XNIVariant> >  XNIVARIANT_ARRAY;
    typedef InstructionData*                                                         LPINSTRUCTIONDATA;
    typedef xarray< LPINSTRUCTIONDATA, CCopyObject<LPINSTRUCTIONDATA>, CDeleteObject<LPINSTRUCTIONDATA> >  INSTRUCTION_ARRAY;
    // eof rt

    typedef XPrim* xprimref;
    typedef xarray<xprimref, CCopyObject<xprimref>, CDeleteObject<xprimref> >        PRIMS_ARRAY;
    
    typedef Attributes* AttributesRef;
    typedef xarray<AttributesRef, CCopyObject<AttributesRef>, CDeleteObject<AttributesRef> >  ATTR_ARRAY;

    #define SYMBOLLIST          SYMBOL_LIST&
    #define BYTEARRAY           BYTE_ARRAY&
    #define DWORDARRAY          DWORD_ARRAY&
    #define STRINGARRAY         STRING_ARRAY&
    #define INTARRAY            INT_ARRAY&
    #define XELEMENT            XortElement&
    #define FileRef             File&
    #define SymbolRef           Symbol&
    #define InstructionDataRef  InstructionData&
    #define TokenRef            Token&
    #define PRIMSARRAY          PRIMS_ARRAY&
    #define XPrimRef            XPrim&
    #define SymbolListInfoRef   SymbolListInfo&
    #define StringRef           string&
    #define ATTRARRAY           ATTR_ARRAY&
    #define INSTRUCTIONARRAY    INSTRUCTION_ARRAY&

#else //----------------------------------------------------------- for XORT compiler
    struct InstructionData
    {
        byte        m_bInstrCode;       // instruction code
        byte        m_bTypeAddr;        // address-operand byte
        XortElement m_Value;            // data for the instruction
        XortElement m_Value2;           // data for the instruction
        string      m_strInstr;         // human readable form of the instruction
        dword       m_dwShiftToNext;    // how many bytes is needed to shift to the next instruction
        dword       m_dwLineNumber;     // the number of line in the text file from where this instruction was generated
    }

    struct XPrim
    {
        byte   m_bCode;
        byte   m_bAddrType;
        string m_csAsm;
    }

    // attributes support. example: [import("proc") name("fopen") module("msvcrt.dll") callconv("cdecl")]
    struct Attributes
    {
        dword  dwAttrCode;
        string csAttrValue;
    }

    #define assert(x) 

    struct Symbol;
    struct File;
    struct Token;
    typedef list<Symbol>    SYMBOL_LIST;
    typedef array<byte>     BYTE_ARRAY;
    typedef array<int>      INT_ARRAY;
    typedef array<dword>    DWORD_ARRAY;
    typedef array<string>   STRING_ARRAY;
    typedef array<Attributes> ATTR_ARRAY;
    typedef array<InstructionData> INSTRUCTION_ARRAY;
    typedef INSTRUCTION_ARRAY INSTRUCTIONARRAY;
    typedef ATTR_ARRAY      ATTRARRAY;
    typedef SYMBOL_LIST     SYMBOLLIST;
    typedef BYTE_ARRAY      BYTEARRAY;
    typedef DWORD_ARRAY     DWORDARRAY;
    typedef STRING_ARRAY    STRINGARRAY;
    typedef INT_ARRAY       INTARRAY;
    typedef XortElement     XELEMENT;
    typedef File            FileRef;
    typedef Symbol          SymbolRef;
    typedef Symbol          symbolref;
    typedef InstructionData InstructionDataRef;
    typedef Token           TokenRef;
    typedef array<XPrim>    PRIMS_ARRAY;
    typedef SymbolListInfo  SymbolListInfoRef;
    typedef string          StringRef;
    typedef Attributes      AttributesRef;

#endif // ---------------------------------------------------------------- CPP_COPMILER

//////////////////////////////////////////////////////////////////////////
// Supported attributes:
const dword attr_import     = 1; // possible values: object, proc
const dword attr_name       = 2; // the name of the entity
const dword attr_module     = 3; // name of the dll
const dword attr_callconv   = 4; // calling convention - for procedures only
const dword attr_platform   = 5; // possible values: windows, osx, linux, ios, android...
const dword attr_export     = 6; // possible values: object, proc. This entity is exported to from the XORT VM to the native dll.

//////////////////////////////////////////////////////////////////////////
// File io related
#include "xfile.h"

enum CallingConvention
{
    cc_unknown = 0,
    cc_xvm,         // means our internal cc
    cc_pascal,      // YES, I mean real PASCAL, not define __stdcall pascall :)))
    cc_cdecl,
    cc_stdcall,
}EOFSTRUCT

// supported storage class definition
const dword cSTORAGE_STACK   = 1;
const dword cSTORAGE_DATA    = 2;
const dword cSTORAGE_MEMBER  = 3;

//////////////////////////////////////////////////////////////////////////
// The SYMBOL - core of the compiler
#ifdef CPP_COMPILER
struct Symbol : public xobject
{
    Symbol() { Zero(); }
    virtual ~Symbol() {
#ifdef __XVM__
        RemoveAllBreakpoints();
#endif
        ;
    }
    IMPLEMENT_SYSTEM_XUNKNOWN()
#else
struct Symbol
{
#endif
    dword        m_SymKind;        // symbol kind: i.e.: scope, var, object, proc...
    string       m_csName;         // symbol name or scope name. For scope names "global" is used, for other - methods names
    STRING_ARRAY m_arAliasNames;   // this is name of the aliases declared via typedef
    dword        m_dwType;         // depends on SymType, this can be treated as return type, variable type, arg type or ignored
    dword        m_dwTemplateType; // the type of build-in template, for now can be initialized only via typedef
    dword        m_dwTemplSpecType;// the type of template type argument
    dword        m_dwFlags;        // const modifier and other flags
    SYMBOL_LIST  m_listMembers;    // method arguments types or object members - local list
    dword        m_dwArgumentsCount; // valid for functions only. 
    //The problem, that m_listMembers is used for stack transitions, thus, 
    //it can not be used to get actual number of arguments during parsing especially for recursive functions!
#ifdef CPP_COMPILER
    SYMBOL_LIST *m_pParent;        // parent list!!! - for search in the next level scope
    Symbol      *m_pOwner;         // pointer to symbol that owns the current symbol - i.e.: object, if Owner is proc - this is local var
#else
    SYMBOL_LIST  m_pParent = NULL;
    Symbol       m_pOwner  = NULL;
#endif
    INSTRUCTION_ARRAY m_Code;      // contains generated xort assembler code - can be valid only in sk_proc symbols
    XortElement  m_Value;          // contains the constant value, or immedate data of last generated instruction if Symbol is procedure
    dword        m_dwLiteralIdx;   // contains literal value
    int          m_iIndex;         // index of the variable in stack or in memory or function ID
    dword        m_dwStorage;      // contains storage class for this symbol
    ATTR_ARRAY   m_arAttributes;   // attributes for procedures: import, calling convention, module name.
    SYMBOL_LIST  m_stackMembers;   // user local variables, used to display debug information at runtime

#ifdef __XVM__
    void        *m_pfnNative;      // pointer to native function, initialized by runtime
    CallingConvention m_CC;        // calling convention for this procedure

    // breakpoint information, valid for procedures only!
    typedef std::map<dword, LPUNKNOWN> MAP_BREAKPOINTS; // stores the IDebugBoundBreakpoint2 objects
    MAP_BREAKPOINTS m_mapBP;        // breakpoints set by DE to this proc, and associated with the XVM IP

    void SetBreakpoint( dword dwInstrIP, LPUNKNOWN pBP ) {
        RemoveBreakpoint( dwInstrIP );
        m_mapBP.insert( MAP_BREAKPOINTS::value_type( dwInstrIP, pBP ) );
        pBP->AddRef();
    }
    void RemoveBreakpoint( dword dwInstrIP ) {
        MAP_BREAKPOINTS::iterator theIter = m_mapBP.find( dwInstrIP );
        if( theIter != m_mapBP.end() ) {
            theIter->second->Release();
            m_mapBP.erase( theIter );
        }
    }
    LPUNKNOWN GetBreakpoint( dword dwInstrIP ) { // no AddRef!
        MAP_BREAKPOINTS::iterator theIter = m_mapBP.find( dwInstrIP );
        if( theIter != m_mapBP.end() ) {
            return theIter->second;
        }
        return NULL;
    }
    void RemoveAllBreakpoints() {
        MAP_BREAKPOINTS::iterator theIter = m_mapBP.begin();
        for( ; theIter != m_mapBP.end(); theIter++ ) {
            theIter->second->Release();
        }
        m_mapBP.clear();
    }

    dword        m_dwNumStatements; // number of statements in the current function

    typedef std::map<dword, dword> MAP_IPTOLINE;
    MAP_IPTOLINE    m_mapIpLine;    // key - IP, value - line number
    void GenerateIPLineMap();       // builds corelation map between IP and source line numbers
    dword GetLineFromIP( dword dwIP ); // returns -1 if there is no such line or valid line number

    void AddIP2LineInfo( dword dwIP, dword dwLine ) { // for extarnal dissasemblers
        m_mapIpLine[ dwIP ] = dwLine;
    }

    dword GetArgumentsCount(); // returns the number of arguments for sk_proc or 0

    lpcstr FindAttribute( dword dwAttrCode ) {
        for( dword i = 0; i < m_arAttributes.count(); i++ ) {
            AttributesRef pAttr = m_arAttributes.get( i );
            if( pAttr->dwAttrCode == dwAttrCode )
                return pAttr->csAttrValue.GetBuffer();
        }
        return NULL;
    }

#endif // __XVM__

    int isNULL() {
        return ( m_SymKind == sk_nullsymbol ) ? 1 : 0;
    }

    void addInstruction( InstructionData PTR instrData ) {
        m_Code.add( instrData );
    }

    void removeLastInstr() { 
        m_Code.del();
    }

    dword getInstrCount() {
        return m_Code.count();
    }
    
    InstructionData PTR getLastInstruction() {
        dword dwCount = m_Code.count();
        if( dwCount > 0 ) {
            return m_Code.get( dwCount - 1 );
        }
        return NULL;
    }

    InstructionData PTR getInstructionByIndex( dword dwIdx ) {
        if( dwIdx < m_Code.count() ) {
            return m_Code.get( dwIdx );
        }
        return NULL;
    }

    void Zero()
    {
        m_SymKind         = sk_nullsymbol;
        m_csName.Empty();
        m_dwType          = 0;
        m_dwTemplateType  = 0;
        m_dwTemplSpecType = 0;
        m_dwFlags         = 0;
        m_pParent         = NULL;
        m_pOwner          = NULL;
        m_iIndex          = 0;
        m_dwStorage       = 0;
        m_dwLiteralIdx    = 0;
        m_listMembers.clear();
        m_dwArgumentsCount= 0;
        m_arAliasNames.clear();
        m_Code.clear();
        m_Value.bType     = 0;
        m_Value.bAccess   = 0;
        m_Value._u8       = 0;
        m_arAttributes.clear();
    }

#ifdef CPP_COMPILER
    void Init( dword sk, string csName, dword dwType, int iIdx, dword dwStorage, SYMBOL_LIST *pParent, Symbol *pOwner, dword dwFlags )
#else
    void Init( dword sk, string csName, dword dwType, int iIdx, dword dwStorage, SYMBOL_LIST pParent, Symbol pOwner, dword dwFlags )
#endif
    {
        m_SymKind         = sk;
        m_csName.Assign( csName );
        m_dwType          = dwType;
        m_dwTemplateType  = 0;
        m_dwTemplSpecType = 0;
        m_dwArgumentsCount= 0;
        m_dwFlags         = dwFlags;
        m_pParent         = pParent; // used for compilation only, lost after save/load
        m_pOwner          = pOwner;  // root object that owns this var or proc or NULL
        m_iIndex          = iIdx;
        m_dwStorage       = dwStorage;
        m_dwLiteralIdx    = 0;
        m_Value.bType     = 0;
        m_Value.bAccess   = 0;
        m_Value._u8       = 0;
    }

    void Copy( SymbolRef Dst )
    {
        Dst.m_SymKind         = m_SymKind;
        Dst.m_csName.Assign( m_csName );
        m_arAliasNames.copy( Dst.m_arAliasNames );
        Dst.m_dwType          = m_dwType;
        Dst.m_dwFlags         = m_dwFlags;
        m_listMembers.copy( Dst.m_listMembers ); // Different semantics C++ - XORT!!!
        Dst.m_dwArgumentsCount= m_dwArgumentsCount;
        //Dst.m_pParent         = m_pParent;  // skip for now
        //Dst.m_Code            = m_Code;
        Dst.m_pOwner          = m_pOwner;
        Dst.m_Value.bType     = m_Value.bType;
        Dst.m_Value.bAccess   = m_Value.bAccess;
        Dst.m_Value._u8       = m_Value._u8;
        Dst.m_dwLiteralIdx    = m_dwLiteralIdx;
        Dst.m_iIndex          = m_iIndex;
        Dst.m_dwStorage       = m_dwStorage;
        m_arAttributes.copy( Dst.m_arAttributes );
    }
}EOFSTRUCT

// The following simple symbol table implementation is based on very simple idea to link all types into
// linked list. For example for the following types:
// 
//  struct M {
//    int A;
//    float B;
//    int Method1( dword a );
//  };
// int Proc( M m );
//
// the following Symbol table will be generated:
//
// --Proc---------M----...
//    |           |    
// struct M m   int a  
//                |
//             float b
//                |
//             Method1---- dword a
//
// The last declared symbol will be first in list - required for simple scope implementation

void      InitSymbolTable( SYMBOLLIST symList );
void      ClearSymbolTable( SYMBOLLIST symList );
void      OpenLocalScope( SYMBOLLIST symList, SymbolRef Proc, int bResetArgIdx );
void      CloseLocalScope( SYMBOLLIST symList, SymbolRef Proc );
SymbolRef FindSymbol( SYMBOLLIST symList, string csName, int bLocalOnly );
SymbolRef FindType( SYMBOLLIST symList, dword dwType );
SymbolRef FindTemplateType( SYMBOLLIST symList, dword dwType, dword dwTemplateSpecType );
SymbolRef FindGlobalVarByIndex( SYMBOLLIST symList, dword dwIdx );
dword     GenNewTypeCode();
void      BuildComplexType( SYMBOLLIST symList, string csName, dword dwBuildInTypeCode, SymbolRef pTemplateType );
// adds new symbol to symbol table, accordingly to current scope generates symbol idx,
// returns newly added symbol

SymbolRef AddSymbol( SYMBOLLIST symList, dword sk, string csName, dword dwType, SYMBOL_LIST PTR pParent, SymbolRef pOwner, dword dwModFlags );
SymbolRef AddStringVarToGlobalScope( dword dwLitPoolIdx ); // adds internal string variable into global scope
void      AddAliasName( SymbolRef pBase, string csName );
void      DumpSymbolTable( SYMBOLLIST symList );
void      DumpStackTransition( SymbolRef Sym );
void      SaveString( string str, FileRef pFile );
void      SaveSymbolTable( SYMBOLLIST symList, FileRef pFile );
int       GenExecutable( SYMBOLLIST symList, string csFileName );
int       LoadExecutable( string csFileName, SYMBOLLIST symList, STRINGARRAY litPool );
void      pushStackVarIdx();
void      popStackVarIdx();
void      ResetLocalIdx();
void      ResetStackVarIdx();
void      ResetArgIdx();
void      RevertArgIndexes( SYMBOLLIST symListArgs );
string    getLocalMarker();
SymbolRef getNULLSymbol();
void      GetSymbolListInfo( SYMBOLLIST symList, SymbolListInfoRef symListInfo );
SYMBOLLIST GetGlobalSymbolList();
void      RemoveRedundandStackTransitionSymbols( SYMBOLLIST symList );

void   lit_Init();
dword  lit_Add( string csLiteral ); // returns the literal index
string lit_Get( dword dwIdx );
void   lit_Free();
void   lit_Dump();
void   lit_DumpLitPool( STRINGARRAY arStr );

//////////////////////////////////////////////////////////////////////////
// io.cpp
void InitErrorTable();
int  io_logOpen( string csFileName );
void io_logClose();
void x_error( dword dwErrorCode, dword dwLine, dword dwPos );
void SetVerbose( int bVerbose );
#ifdef CPP_COMPILER
void message( const char *lpszMsg, ... );
void OutMsg( const char *lpszMsg, ... );
#else
void message( string lpszMsg, ... );
void OutMsg( string lpszMsg, ... );
#endif

//////////////////////////////////////////////////////////////////////////
// lex.cpp
struct Token
{
    dword     m_TT;             // type of the token
    dword     m_dwTokenCode;    // internal token code
    dword     m_dwLine;         // line number in text file
    dword     m_dwPos;          // position in text file
    XortElement m_Val;          // constant token value if any
    string    m_csVal;          // ident name, string, etc...
    dword     m_dwLiteralIdx;   // index of the literal

    void Clear()
    {
        m_TT             = tt_unknown;
        m_dwTokenCode    = tk_unknown;
        m_dwLine         = 0;
        m_dwPos          = 0;
        m_Val.bType      = 0;
        m_Val.bAccess    = 0;
        m_Val._u8        = 0;
        m_csVal.Empty();
        m_dwLiteralIdx   = 0;
    }

    void Copy( TokenRef Dst )
    {
        Dst.m_TT             = m_TT;
        Dst.m_dwTokenCode    = m_dwTokenCode;
        Dst.m_dwLine         = m_dwLine;
        Dst.m_dwPos          = m_dwPos;
        Dst.m_Val.bType      = m_Val.bType;
        Dst.m_Val.bAccess    = m_Val.bAccess;
        Dst.m_Val._u8        = m_Val._u8;
        Dst.m_csVal.Assign( m_csVal );
        Dst.m_dwLiteralIdx   = m_dwLiteralIdx;
    }
#ifdef CPP_COMPILER
    void operator= ( const Token& tok )
    {
        m_TT             = tok.m_TT;
        m_dwTokenCode    = tok.m_dwTokenCode;
        m_dwLine         = tok.m_dwLine;
        m_dwPos          = tok.m_dwPos;
        m_Val.bType      = tok.m_Val.bType;
        m_Val.bAccess    = tok.m_Val.bAccess;
        m_Val._u8        = tok.m_Val._u8;
        m_csVal.Assign( tok.m_csVal );
        m_dwLiteralIdx   = tok.m_dwLiteralIdx;
    }
    void Dump() {
        DbgPrint( "Token '%@' type: %@, Code: %@, Line: 0x%@, Pos: 0x%@", m_csVal.GetBuffer(), m_TT, m_dwTokenCode, m_dwLine, m_dwPos );
    }
#else
    void Dump() {
        DbgPrint( "Token '%@' type: %@, Code: %@, Line: 0x%@, Pos: 0x%@", m_csVal, m_TT, m_dwTokenCode, m_dwLine, m_dwPos );
    }
#endif
}EOFSTRUCT

//////////////////////////////////////////////////////////////////////////
// lex.cpp
int lex_Input( string csFile );
Token lex_GetToken( SYMBOLLIST symList );
Token lex_GetLastToken();
void lex_PutBack();
int lex_match( SYMBOLLIST symList, dword tokType, dword dwTokCode, int bTestLastTok );
int lex_ConfirmMember( SYMBOLLIST symList, SymbolRef Var, SymbolRef ObjBase ); // method returns 1 if current token is the object member, 0 otherwise
string lex_getFileName();
string lex_GetLine( dword dwIdx );

//////////////////////////////////////////////////////////////////////////
// utils.cpp
int    x_isdigit( byte ch );
dword  x_ishexdigit( byte ch );
dword  x_getHexValue( byte ch );
int    x_isalpha( byte c );
int    x_isalnum( byte c );
int    x_isdelim( byte c );
int    x_isfmtstr( StringRef csFmt );
dword  x_gettypesize( dword dwType );
string x_type2str( byte bType );
void   x_copyelement( XELEMENT dst, XELEMENT src );
int    x_isAssignOp( dword op );
void   x_copyCode( SymbolRef Dst, SymbolRef Src );
byte   x_code_from_tok( dword op );
int    x_convert_type( dword bType, XELEMENT x, XELEMENT y );
int    x_findAttribute( ATTRARRAY Attr, dword dwCode, StringRef strOut );
int    x_is_metainstruction( byte bCode );

#ifdef __XVM__
string xEscapeString( StringRef str ); // escape special characters
string xFormatElementValue( XELEMENT x, dword dwRadix );
inline byte x_clamp_type( dword dwType ) { return ( dwType > t_p ) ? t_p : (byte)dwType; }
#endif

//////////////////////////////////////////////////////////////////////////
// syntax.cpp
int do_object( SYMBOLLIST symList, ATTRARRAY arAttributes, StringRef csNameOut );
int do_var_proc( SYMBOLLIST symList, dword dwTypeCode, SymbolRef Func, dword dwFlags, StringRef csNameOut );
int do_callproc( SYMBOLLIST symList, SymbolRef Func, SymbolRef ProcToCall, int bInExpr, SymbolRef Object );
int do_main_statement( SYMBOLLIST symList, SymbolRef Func, int bMainScope, word wStartLabel, word wExitLabel, dword dwNumEntries, int bSingle, dword dwStmtCode );
int do_typedef( SYMBOLLIST symList );
int do_enum( SYMBOLLIST symList );
int do_attributes( SYMBOLLIST symList, ATTRARRAY arAttributes );
int validate_obj_attributes( ATTRARRAY arAttributes );
int validate_native_type( SYMBOLLIST symList, dword dwTypeCode );
int isObjectImported( SymbolRef obj );

//////////////////////////////////////////////////////////////////////////
// expr.cpp
// - SymbolRef Func param is the function where the code code will be generated,
// or nullSymbol in case the constant expression expected
int get_expr(SYMBOLLIST symList, SymbolRef vResult, SymbolRef Func, SymbolRef vTargetResult, int bInCondExp ); // if pFunc == NULL constant expression expected!
int exp_mem_ptr( SYMBOLLIST symList, SymbolRef vResult, SymbolRef pFunc, SymbolRef Var );
int const_arith( dword op, XELEMENT x, XELEMENT y ); // calculates the contsnat arith between 2 operands: x op= y;

//////////////////////////////////////////////////////////////////////////
// codegen.cpp
void codegen_Init();
void codegen_Uninit();
word get_CurLabel();
word get_NextLabel();
int gen_load( SymbolRef Func, SymbolRef Src );
int gen_loadZero( SymbolRef Func, dword dwType, int bCreateObject );
int gen_loadConst( SymbolRef Func, XELEMENT Val );
int gen_loadConstVar( SymbolRef Func, SymbolRef Src );
int gen_neg( SymbolRef Func, SymbolRef Dst );
int gen_not( SymbolRef Func, SymbolRef Dst );
int gen_lognot( SymbolRef Func, SymbolRef Dst );
int gen_arith( byte bCode, SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_cmp( byte bCode, SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_logic_arith( byte bCode, SymbolRef Func, SymbolRef Dst, SymbolRef Src, SymbolRef Result );
int gen_assign( SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_call( SymbolRef Func, SymbolRef ProcToCall );
int gen_invoke( SymbolRef Func, SymbolRef ProcToCall );
int gen_invoke_ti( SymbolRef Func, SymbolRef ProcToCall );
int gen_ret( SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_ret2( SymbolRef Func, SymbolRef Dst );
int gen_dup( SymbolRef Func );
int gen_drop( SymbolRef Func );
int gen_convert( SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_ifnz( SymbolRef Func, SymbolRef Dst, word wLblIdx );
int gen_ifz( SymbolRef Func, SymbolRef Dst, word wLblIdx );
int gen_ifneq( SymbolRef Func, SymbolRef Dst, SymbolRef Src, word wLblIdx );
int gen_label( SymbolRef Func, word wLblIdx );
int gen_goto( SymbolRef Func, word wLblIdx );
int gen_enter( SymbolRef Func );
int gen_leave( SymbolRef Func, int bDecStkIdx );
int gen_member_get( SymbolRef Func, SymbolRef Object, SymbolRef Member );
int gen_member_put( SymbolRef Func, SymbolRef Member, SymbolRef Src );
int gen_convert( SymbolRef Func, SymbolRef Dst, SymbolRef Src );
int gen_inc_dec( byte bCode, SymbolRef Func, SymbolRef Target );
int gen_inc_dec_onStackTop( byte bCode, SymbolRef Func, SymbolRef Target );
int gen_nz( SymbolRef Func, SymbolRef Target );
int gen_pack( SymbolRef Func, SymbolRef Target );
int gen_unpack( SymbolRef Func, SymbolRef Target );
// meta instructions
void gen_StackVarMark( SymbolRef Func, word wIdx );
void gen_StackVarEnd( SymbolRef Func, word wIdx );
void gen_BeginArg( SymbolRef Func, word wIdx );
void gen_EndArg( SymbolRef Func, word wIdx );
void gen_BeginCall( SymbolRef Func, word wIdx );
void gen_EndCall( SymbolRef Func, word wIdx );
void gen_BeginIf( SymbolRef Func, word wIdx );
void gen_EndIf( SymbolRef Func, word wIdx );
void gen_BeginRet( SymbolRef Func, word wIdx );
void gen_EndRet( SymbolRef Func, word wIdx );
void gen_BeginAssign( SymbolRef Func, word wIdx );
void gen_EndAssign( SymbolRef Func, word wIdx );
void gen_BeginSwitch( SymbolRef Func, word wIdx );
void gen_EndSwitch( SymbolRef Func, word wIdx );
void gen_BeginCase( SymbolRef Func, word wIdx );
void gen_EndCase( SymbolRef Func, word wIdx );

//////////////////////////////////////////////////////////////////////////
// optimizer.cpp
int optimize_program( SYMBOLLIST symList );

//////////////////////////////////////////////////////////////////////////
// dasm.cpp
int decode_instr( BYTEARRAY buf, dword dwStartFrom, InstructionDataRef decData );
int decode_instr_fast( BYTEARRAY buf, dword dwStartFrom, InstructionDataRef decData );
int decode_instr_text( InstructionDataRef decData );

//////////////////////////////////////////////////////////////////////////
// main.cpp
const dword XCL_OPTIMIZATION_ON         = 0x00000001; // controls the full xort code optimization
const dword XCL_META_INSTRUCTIONS_ON    = 0x00000020; // controls helper meta (pseudo) instructions generation for jit
void  SetCompilationFlags( dword dwFlags );
dword GetCompilationFlags();

//////////////////////////////////////////////////////////////////////////
// xprim.cpp
void xInitPrims();
void xFreePrims();
string xPrimFindAsmString( byte bCode, byte bTypeAddr );

//////////////////////////////////////////////////////////////////////////
// XVM related functions
#ifdef __XVM__
// Performs checking if the text position is inside code block, 
// returns: proc-symbol and can move requested line number to the nearest valid line
// if line doesn't correspond to code, NULL is returned
SymbolRef ValidateBreakpointPosition( SYMBOLLIST symList, dword dwLineNo, dword *pdwLineNoOut );

// Returns the IP of the first instruction (offset in code buffer) for the specified line position.
// In most cases this will be the begin of the statement or expression.
dword GetInstructionIPFromSourceLine( SymbolRef symProc, dword dwLineNo );

#endif